package io.github.biezhi.elves.spider;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.math.BigDecimal;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.ListIterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.alibaba.fastjson.JSONObject;

import io.github.biezhi.elves.bidding.model.BiddingKeywordDTO;
import io.github.biezhi.elves.bidding.model.DownloadFileDTO;
import io.github.biezhi.elves.common.Constants;
import io.github.biezhi.elves.common.FileType;
import io.github.biezhi.elves.fsggzy.model.FsBiddingList;
import io.github.biezhi.elves.fsggzy.model.FsDownloadBiddingHtml;
import io.github.biezhi.elves.fsggzy.model.FsRecordBiddingNotice;
import io.github.biezhi.elves.ok.http.FastHttpClient;
import io.github.biezhi.elves.service.BidService;
import io.github.biezhi.elves.service.FsggzyService;
import io.github.biezhi.elves.utils.StringUtils;

/**
 * 佛山公共资源交易网爬取(http://www.fsggzy.cn/)
 * @author Administrator
 *
 */
@Component
public class FsggzySpider {
	@Autowired
	private FsggzyService fsggzyService;
	@Autowired
	private BidService bidService;
	
	//休眠时间
	Long sleepTime = 1000L;
	//页码，默认第一页
	static Integer pageIndex = 1;
	//是否完成保存记录投标网公告标题数据
	static Boolean finished = false;
	static String channelid = "59154";

	//record表爬取网址
	String bidUrl = "http://219.130.221.60:8084/was5/web/search";
	/**
	 * ========================通过搜索关键字爬取开始================================================================
	 */
	//保存所有关键字信息到数据库
	public void saveFsRecord() throws Exception{
		List<BiddingKeywordDTO> keywords = bidService.listBiddingKeyword(1);
		for (BiddingKeywordDTO biddingKeywordDTO : keywords) {
			if (biddingKeywordDTO.getSearch().equals("T")) {
				String keyword = biddingKeywordDTO.getKeyword();
				//获取该关键字查询总页数
				System.out.println(keyword);
				Integer lastPageIndex = getLastPageIndexByKeyword(keyword);
				saveFsRecordByOnekey(keyword,lastPageIndex);
			}
		}
	}
	
	private Integer getLastPageIndexByKeyword(String keyword) {
		
		String htmlContent = getHtmlContent(keyword,channelid,pageIndex,bidUrl);
		if (htmlContent != null && htmlContent.length()>0) {
			//解析文档
			Document doc = Jsoup.parse(htmlContent);
			String url = doc.select(".last-page").attr("href");
			System.out.println(url);
			if (url != null && url.length()>0) {
				//indexOf首次出现特殊符号的位置
				String lastIndex = url.substring(url.indexOf("=")+1, url.indexOf("&"));
				System.out.println(lastIndex);
				return Integer.valueOf(lastIndex);
			}
		}
		return 1;
	}

	//根据一个关键字保存FsDownloadBiddingHtml对象
	public void saveFsRecordByOnekey(String keyword,Integer lastPageIndex){
		
		for (pageIndex = 1; pageIndex <= lastPageIndex; pageIndex++) {
			//获取整个网页信息
			String htmlContent = getHtmlContent(keyword,channelid,pageIndex,bidUrl);
			if (htmlContent != null && htmlContent.length()>0) {
			//解析文档
			Document doc = Jsoup.parse(htmlContent);
			Elements elements = doc.select(".secondrightlistbox");
			//System.out.println(elements);
			Elements lis = elements.select("li");
			for (Element li : lis) {
				FsRecordBiddingNotice fsRecordBiddingNotice = new FsRecordBiddingNotice();
				String text = li.text();
				//获取发布日期
				String pubdate = text.substring(text.indexOf("[")+1, text.indexOf("]"));
				//获取公告标题
				String title = text.substring(text.indexOf("]")+1, text.length());
				//获取公告详情链接
				String noticeDetailUrl = li.select("a").attr("href");
				//获取公告类型
				String noticeType="";
				if (noticeDetailUrl.contains("/zfcg/zf_zbxx/") || noticeDetailUrl.contains("/qycg/jggg/") || noticeDetailUrl.contains("/yxcg/yx_zbgg/")) {
					noticeType = "中标公告";
				}else if (noticeDetailUrl.contains("/zfcg/zf_cggsgg/") || noticeDetailUrl.contains("/qycg/cggg/")  || noticeDetailUrl.contains("/yxcg/yx_yxcg/")) {
					noticeType ="采购公告";
				}else if (noticeDetailUrl.contains("/zfcg/zf_gzgg/") || noticeDetailUrl.contains("/qycg/gzgg/")  || noticeDetailUrl.contains("/yxcg/yx_gzgg/")) {
					noticeType ="更正公告";
				}
				
//				System.out.println(noticeType);
				fsRecordBiddingNotice.setNoticeType(noticeType);
				fsRecordBiddingNotice.setKeyword(keyword);
				fsRecordBiddingNotice.setNoticeDetailUrl(noticeDetailUrl);
				fsRecordBiddingNotice.setPubdate(pubdate);
				fsRecordBiddingNotice.setSize(pageIndex);
				fsRecordBiddingNotice.setTitle(title);
				
				//根据数据库是否有这个noticeDetailUrl来判断条信息是否已经保存过，保存过就不再保存了
				Integer count = fsggzyService.countByUrl(noticeDetailUrl);
				if (count == 0) {
					//保存fsRecordBiddingNotice到数据库
					fsggzyService.saveFsRecordBiddingNotice(fsRecordBiddingNotice);
				}
			}
			}
			sleepTime();
			System.out.println(pageIndex);
			if (pageIndex==lastPageIndex) {
				System.out.println(keyword+"关键字执行完成！");
			}
		}
	}
	//get请求获取Html页面
		private String getHtmlContent(String searchword, String channelid, Integer pageIndex,String bidUrl) {
			String result = "";
			try {
				io.github.biezhi.elves.ok.http.Response okHttpResponse;
				Proxy proxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(Constants.PROXY_IP, Constants.PROXY_PORT));
				okHttpResponse = FastHttpClient
						.newBuilder()
						.proxy(proxy)
						.build()
						.get()
						.addParams("channelid", channelid)
						.addParams("page", pageIndex.toString())
						.addParams("searchword", searchword).url(bidUrl)
						.build()
						.execute();
				
				result = okHttpResponse.string();
				//System.out.println(result);
				if(!StringUtils.isNullOrEmpty(result)){
					return result;
				}
			} catch (SocketTimeoutException e) {
				System.err.println("===连接超时，重新调用postHtmlContent()方法===");
				getHtmlContent(searchword, channelid, pageIndex,bidUrl);
			} catch (Exception e) {
				e.printStackTrace();
				System.err.println("===其他异常，重新调用postHtmlContent()方法===");
				getHtmlContent(searchword, channelid, pageIndex,bidUrl);
			}
			return result;
		}
		
		private String getHtmlContent(String url) {
			String result = "";
			try {
				io.github.biezhi.elves.ok.http.Response okHttpResponse;
				Proxy proxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(Constants.PROXY_IP, Constants.PROXY_PORT));
				okHttpResponse = FastHttpClient
						.newBuilder()
						.proxy(proxy)
						.build()
						.get()
						.url(url)
						.build()
						.execute();
				
				result = okHttpResponse.string();
				//System.out.println(result);
				if(!StringUtils.isNullOrEmpty(result)){
					return result;
				}
			} catch (SocketTimeoutException e) {
				System.err.println("===连接超时，重新调用postHtmlContent()方法===");
				getHtmlContent(url);
			} catch (Exception e) {
				e.printStackTrace();
				System.err.println("===其他异常，重新调用postHtmlContent()方法===");
				getHtmlContent(url);
			}
			return result;
		}

/**
 * 	==========================================通过查询关键字爬取结束=================================================

 */
	
	
	private void sleepTime() {
		try {
			Thread.sleep(sleepTime);
			System.err.println("休眠" + sleepTime + "毫秒后，往下执行");
		} catch (Exception e){}
	}
	
	private void sleepTime(Long sleepTime) {
		try {
			Thread.sleep(sleepTime);
			System.err.println("休眠" + sleepTime + "毫秒后，往下执行");
		} catch (Exception e){}
	}
	
	//读取record表中的数据，根据notice_detail_url查询公告内容，并保存到t_download_fs
	public void saveFsDownload(){
		//查询record表中notice_type等于中标公告并且发布日期在2018年的数据
		String noticeType = "中标公告";
		String date = "2018";
		List<FsRecordBiddingNotice> listFsRecordBiddingNotice = fsggzyService.findRecordByNoticeTypeAndDate(noticeType,date);
		
		//查询出符合条件所有的notice_detail_url
		for (FsRecordBiddingNotice fsRecordBiddingNotice : listFsRecordBiddingNotice) {
			//定义一个FsDownloadBiddingHtml对象，将数据存入对象
			FsDownloadBiddingHtml FsDownloadBiddingHtml = new FsDownloadBiddingHtml();
			
			String noticeDetailUrl = fsRecordBiddingNotice.getNoticeDetailUrl();
			if (noticeDetailUrl!=null && noticeDetailUrl.length()>0) {
				String htmlContent = getHtmlContent(noticeDetailUrl);
				String keyword = fsRecordBiddingNotice.getKeyword();
				String pubdate = fsRecordBiddingNotice.getPubdate();
				
				FsDownloadBiddingHtml.setHtmlContent(htmlContent);
				FsDownloadBiddingHtml.setKeyword(keyword);
				FsDownloadBiddingHtml.setNoticeDetailUrl(noticeDetailUrl);
				FsDownloadBiddingHtml.setPubdate(pubdate);
				FsDownloadBiddingHtml.setNoticeType(noticeType);
				
				//在文档里面查找<a>标签
				Document doc = Jsoup.parse(htmlContent);
				//查找正文中所有a标签
				Elements allAElements = doc.select("a");
				ListIterator<Element> htmlAResultIter = allAElements.listIterator();
				
				List<DownloadFileDTO> listDownloadFile = new ArrayList<DownloadFileDTO>();
				while(htmlAResultIter.hasNext()) {
					Element get = htmlAResultIter.next();
					String fileName = get.text();
					String fileUrl = get.attr("href");
					//文件后缀
					String[] fileSuffixs = FileType.FILE_SUFFIX.split(",");
					for(String fileSuffix : fileSuffixs) {
						//判断获取的文件链接与文件后缀是否匹配
						if(fileUrl.endsWith(fileSuffix)) {
							DownloadFileDTO downloadFile = new DownloadFileDTO();
							downloadFile.setFileName(fileName);
							downloadFile.setFileUrl(fileUrl);
							
							System.out.println("fileName：" + fileName);
							System.out.println("fileUrl：" + fileUrl);
							listDownloadFile.add(downloadFile);
							break;
						}
					}
					FsDownloadBiddingHtml.setDownloadFile(JSONObject.toJSONString(listDownloadFile));
				}
				
				
				//查询这条公告在数据库是否存在
				Integer count = fsggzyService.countByDownloadUrl(noticeDetailUrl);
				if (count==0) {
					//保存FsDownloadBiddingHtml对象到t_download_fs表中
					fsggzyService.saveFsDownloadBiddingHtml(FsDownloadBiddingHtml);
				}
			}
			
		}
		System.out.println("插入完成！");
	}
	
	
	//解析文档
	public void resolveHtmlContent(){
		//读取数据库 获取文档信息
		List<FsDownloadBiddingHtml> listFsDownloadBiddingHtml = fsggzyService.listDownloadHtml();
		if (!listFsDownloadBiddingHtml.isEmpty()) {
			for (FsDownloadBiddingHtml fsDownloadBiddingHtml : listFsDownloadBiddingHtml) {
				String htmlContent = fsDownloadBiddingHtml.getHtmlContent();
				String noticeDetailUrl = fsDownloadBiddingHtml.getNoticeDetailUrl();
				if (htmlContent != null && !htmlContent.equals("")) {
					//解析htmlContent,返回FsbiddingList对象
					FsBiddingList fsbiddingList = resolveToFsbiddingList(htmlContent);
					//公告详情内容链接存入对象
					fsbiddingList.setBidUrl(noticeDetailUrl);
					//插入到数据库
					fsggzyService.saveFsbiddingList(fsbiddingList);
				}
			}
		}
		System.out.println("文档字段插入完成！");
	}
	
	
	private FsBiddingList resolveToFsbiddingList(String htmlContent){
		FsBiddingList fsBiddingList = new FsBiddingList();
//		htmlContent = "";
//		htmlContent = getXmlString();
		Document doc = Jsoup.parse(htmlContent);
		//获取公告标题
		String noticeTitle = doc.select(".contenttitle2 > h3").text();
		System.out.println("标题："+noticeTitle);
		fsBiddingList.setNoticeTitle(noticeTitle);
		//获取发布日期
		String pubdate = doc.select(".fbtime").text();
		System.out.println("发布日期:"+pubdate);
		fsBiddingList.setPubdate(pubdate);
		//城市
		String city = "佛山";
		System.out.println("城市："+city);
		fsBiddingList.setCity(city);
		
		if (doc.select(".content2") !=null) {
			String content2 = doc.select(".content2").text();
			System.out.println(content2);
			//项目id
			String prjId = "";
			String utilRegex = "一、采购项目编号：[\\s\\S]+二、采购项目名称";
			Pattern pattern = Pattern.compile(utilRegex);
			Matcher matcher = pattern.matcher(content2);
			while (matcher.find()) {
				prjId = matcher.group();
				prjId = prjId.replace("一、采购项目编号：", "");
				prjId = prjId.replace("二、采购项目名称", "");
			}
			if (prjId.length()>100) {
				prjId="";
			}
			System.out.println("项目ID："+prjId);
			fsBiddingList.setPrjId(prjId);
			//供应商
			String supplier = "";
			String utilRegex2 = "中标供应商名称：[\\s\\S]+法人代表";
			Pattern pattern2 = Pattern.compile(utilRegex2);
			Matcher matcher2 = pattern2.matcher(content2);
			while (matcher2.find()) {
				supplier = matcher2.group();
				supplier = supplier.replace("中标供应商名称：", "");
				supplier = supplier.replace("法人代表", "");
			}
			if (StringUtils.isNullOrEmpty(supplier)) {
				utilRegex2 = "中标供应商名称[\\s\\S]+法人代表";
				pattern2 = Pattern.compile(utilRegex2);
				matcher2 = pattern2.matcher(content2);
				while (matcher2.find()) {
					supplier = matcher2.group();
					supplier = supplier.replace("中标供应商名称", "");
					supplier = supplier.replace("法人代表", "");
				}
			}
			System.out.println("供应商:"+supplier);
			if (supplier.length()>100) {
				supplier="";
			}
			fsBiddingList.setSupplier(supplier);
			//项目名称
			String prjName = "";
			String utilRegex3 = "二、采购项目名称：[\\s\\S]+三、采购项目预算金额";
			Pattern pattern3 = Pattern.compile(utilRegex3);
			Matcher matcher3 = pattern3.matcher(content2);
			while (matcher3.find()) {
				prjName = matcher3.group();
				prjName = prjName.replace("二、采购项目名称：", "");
				prjName = prjName.replace("三、采购项目预算金额", "");
			}
			if (prjName.length()>100) {
				prjName="";
			}
			System.out.println("项目名称："+prjName);
			fsBiddingList.setPrjName(prjName);
			//中标时间
			String dealTime = "";
			String utilRegex4 = "评审日期：[\\s\\S]+评审地点";
			Pattern pattern4 = Pattern.compile(utilRegex4);
			Matcher matcher4 = pattern4.matcher(content2);
			while (matcher4.find()) {
				dealTime = matcher4.group();
				dealTime = dealTime.replace("评审日期：", "");
				dealTime = dealTime.replace("评审地点", "");
			}
			if (dealTime.length()>100) {
				dealTime="";
			}
			System.out.println("中标时间:"+dealTime);
			fsBiddingList.setDealTime(dealTime);
			//预算金额
			String budget = "";
			String utilRegex5 = "预算金额（元）：[\\s\\S]+四、采购方式";
			Pattern pattern5 = Pattern.compile(utilRegex5);
			Matcher matcher5 = pattern5.matcher(content2);
			while (matcher5.find()) {
				budget = matcher5.group();
				budget = budget.replace("预算金额（元）：", "");
				budget = budget.replace("四、采购方式", "");
			}
			if (StringUtils.isNullOrEmpty(budget)) {
				utilRegex5 = "采购项目预算金额：[\\s\\S]+四、";
				pattern5 = Pattern.compile(utilRegex5);
				matcher5 = pattern5.matcher(content2);
				while (matcher5.find()) {
					budget = matcher5.group();
					budget = budget.replace("采购项目预算金额：", "");
					budget = budget.replace("四、", "");
				}
			}
			System.out.println("预算金额："+budget);
			fsBiddingList.setBudget(budget);
			//采购方式
			String buyType = "";
			String utilRegex6 = "四、采购方式：[\\s\\S]+五、中标供应商";
			Pattern pattern6 = Pattern.compile(utilRegex6);
			Matcher matcher6 = pattern6.matcher(content2);
			while (matcher6.find()) {
				buyType = matcher6.group();
				buyType = buyType.replace("四、采购方式：", "");
				buyType = buyType.replace("五、中标供应商", "");
			}
			System.out.println("采购方式："+buyType);
			fsBiddingList.setBuyType(buyType);
			//获取客户（发标、采购方）
			String customer = "";
			String utilRegex7 = "受[\\s\\S]+的委托";
			Pattern pattern7 = Pattern.compile(utilRegex7);
			Matcher matcher7 = pattern7.matcher(content2);
			while (matcher7.find()) {
				customer = matcher7.group();
				customer = customer.replace("受", "");
				customer = customer.replace("的委托", "");
			}
			System.out.println("客户（发标、采购方）："+customer);
			fsBiddingList.setCustomer(customer);
			//发标时间
			String publishTime = "";
			String utilRegex8 = "于[\\s\\S]+日就";
			Pattern pattern8 = Pattern.compile(utilRegex8);
			Matcher matcher8 = pattern8.matcher(content2);
			while (matcher8.find()) {
				publishTime = matcher8.group();
				publishTime = publishTime.replace("于", "");
				publishTime = publishTime.replace("就", "");
			}
			if (publishTime.length()>100) {
				publishTime="";
			}
			System.out.println("发标时间:"+publishTime);
			fsBiddingList.setPublishTime(publishTime);
		}
		//成交额
/*		if (doc.select(".MsoNormalTable tr") != null && doc.select(".MsoNormalTable tr").size()>1 && 
				doc.select(".MsoNormalTable tr").get(1).select("td")!=null && 
						doc.select(".MsoNormalTable tr").get(1).select("td").size()>5) {
			
			String bidPrice = doc.select(".MsoNormalTable tr").get(1).select("td").get(5).text();
		System.out.println("成交额:"+bidPrice);
			fsBiddingList.setBidPrice(bidPrice);
		}*/
		if (doc.select("table") != null && doc.select("table").size()>0 
				&& doc.select("table").get(0).select("tr") != null 
				&& doc.select("table").get(0).select("tr").size()>1 
				&& doc.select("table").get(0).select("tr").get(1).select("td") != null
				&& doc.select("table").get(0).select("tr").get(1).select("td").size() >5) {
			String bidPrice = doc.select("table").get(0).select("tr").get(1).select("td").get(5).text();
			System.out.println("成交额:"+bidPrice);
			fsBiddingList.setBidPrice(bidPrice);
		}
		
		return fsBiddingList;
	}

	//读取文本为字符串
	public String getXmlString() {
		String xmlString;
		char[] strBuffer = null;
		int flen = 0;
		File xmlfile = new File("D:\\fsHtml2.txt");
		try {
		FileInputStream input = new FileInputStream(xmlfile);
		InputStreamReader in=new InputStreamReader(input,"utf-8");//字节流字符流转化的桥梁
		flen = (int)xmlfile.length();
		strBuffer = new char[flen];
		in.read(strBuffer, 0, flen);
		} catch (FileNotFoundException e) {
		e.printStackTrace();
		} catch (IOException e) {
		e.printStackTrace();
		}
		xmlString = new String(strBuffer); //构建String时，可用char[]类型，

		return xmlString;
		}
	
	public static void main(String[] args) throws Exception {
		FsggzySpider fsggzySpider = new FsggzySpider();
		/*Integer pageIndex = 1;
		String searchword = "大数据";
		String channelid = "59154";
		fsggzySpider.saveFsRecord();*/
		//fsggzySpider.getLastPageIndexByKeyword("数据库");
		fsggzySpider.resolveToFsbiddingList("");
	}

}
